package gbench.appdemo.crawler;

import org.jsoup.Jsoup;
import org.junit.jupiter.api.Test;
import static gbench.common.client.AbstractHttpClient.*;
import gbench.common.fs.FileSystem;
import gbench.commonApp.data.tushare.TuShareApi;
import static gbench.common.tree.LittleTree.IRecord.*;
import java.text.MessageFormat;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Stream;

/**
 * 
 * @author gbench
 *
 */
public class ErShiSiShiCrawler {
    
    /**
     * 
     * @param url
     * @param book
     */
    final void download(String url,String book) {
        final var api = new TuShareApi(url,"");
        final var latch = new CountDownLatch(1);
        api.ajax(REC("$method","get","$success_type","HR2U",
            "$success", (HR2U<String>)response -> {
                final var body_pattern = Pattern.compile("<div class=\"content-box mauto\">(.+)</div>");
                try {
                    final String page = str(response);// 获取网页信息
                    final var body_matcher = body_pattern.matcher(page);
                    if(body_matcher.find()) {
                        final var html = body_matcher.group(1);// 提取文档
                        final var doc = Jsoup.parse(html);
                        final var title = doc.getElementsByTag("body").get(0).ownText();// 提取文章标题
                        //写入html文件
                        FileSystem.write(MessageFormat.format("{0}/{1}/html/{2}.html", this.home,book,title),page);
                        
                        final var id_matcher = Pattern.compile(".*/([^/]+)\\.html").matcher(url);
                        id_matcher.find();
                        final var id = id_matcher.group(1);
                        System.out.println("title:"+title);
                        final var ai = new AtomicInteger();
                        doc.getElementsByTag("p").forEach(elem->{
                            final var line = elem.html();
                            System.out.println(ai.getAndIncrement()+")"+elem.html());
                            FileSystem.write(MessageFormat.format("{0}/{1}/"+id+"_{2}/p{3}.html", this.home,book,title,ai.get()),line);
                        });
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                latch.countDown();
             return null;
        }));
        try {
            latch.await();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    public String home = 
            //"D:\\sliced\\ebook\\texts";
          "D:\\sliced\\ebook\\texts2";
    
    /**
     * 史记
     */
    @Test
    public void shiji() {
        Stream.iterate(2560, i->i<=2689,i->i+1)
        .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
        .forEach(url->download(url,"史记"));
    }
    
    /**
     * 汉书
     */
    @Test
    public void hanshu() {
        Stream.iterate(2979, i->i<=3098,i->i+1)
        .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
        .forEach(url->download(url,"汉书"));
    }
    
    /**
     * 后汉书
     */
    @Test
    public void houhanshu() {
        Stream.iterate(2979, i->i<=3098,i->i+1)
        .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
        .forEach(url->download(url,"后汉书"));
    }
    
    /**
     * 三国志
     */
    @Test
    public void sanguozhi() {
        Stream.iterate(4156, i->i<=4221,i->i+1)
        .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
        .forEach(url->download(url,"三国志"));
    }
    
    /**
     * 晋书 
     */
    @Test
    public void jinshu() {
        Stream.iterate(4290, i->i<=4419,i->i+1)
        .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
        .forEach(url->download(url,"晋书"));
    }
    
    /**
     *  全书下载
     */
    @Test
    public void download() {
        final var dirs = A(
//                REC("id",1,"name","史记","start",2560,"end",2680,"author","司马迁"),
//                REC("id",2,"name","汉书","start",2979,"end",3098,"author","班固"),
//                REC("id",3,"name","后汉书","start",3309,"end",3438,"author","范晔"),
//                REC("id",4,"name","三国志","start",4156,"end",4221,"author","陈寿"),
//                REC("id",5,"name","晋书","start",4290,"end",4419,"author","房玄龄等"),
//                REC("id",6,"name","宋书","start",4780,"end",4673,"author","梁沈约"),
                REC("id",7,"name","南齐书","start",4574,"end",4839,"author","萧子显"),
                REC("id",8,"name","梁书","start",4913,"end",4969,"author","姚察，姚思廉"),
                REC("id",9,"name","陈书","start",5057,"end",5093,"author","姚思廉"),
                REC("id",10,"name","魏书","start",5204,"end",5334,"author","魏收"),
                REC("id",11,"name","北齐书","start",5522,"end",5571,"author","李百药"),
                REC("id",12,"name","周书","start",8631,"end",8681,"author","令狐德棻等"),
                REC("id",13,"name","隋书","start",8843,"end",8928,"author","魏征"),
                REC("id",14,"name","南史","start",9097,"end",9176,"author","李延寿"),
                REC("id",15,"name","北史","start",9425,"end",9524,"author","李大师、李延寿"),
                REC("id",16,"name","旧唐书","start",10027,"end",10241,"author","赵莹、刘昫等"),
                REC("id",17,"name","新唐书","start",10792,"end",11040,"author","宋祁、欧阳修等"),
                REC("id",18,"name","旧五代史","start",11537,"end",11687,"author","薛居正、卢多逊等"),
                REC("id",19,"name","新五代史","start",11860,"end",11934,"author","欧阳修"),
                REC("id",20,"name","宋史","start",12073,"end",12665,"author","脱脱、阿鲁图"),
                REC("id",21,"name","辽史","start",13181,"end",13297,"author","脱脱等"),
                REC("id",22,"name","金史","start",13460,"end",13595,"author","脱脱"),
                REC("id",23,"name","元史","start",13772,"end",13982,"author","宋濂、王袆"),
                REC("id",24,"name","明史","start",14436,"end",14768,"author","张廷玉、万斯同等")
        );
        
        Stream.of(dirs).forEach(e->{
            Stream.iterate(e.i4("start"), i->i<=e.i4("end"),i->i+1)
            .map(i->MessageFormat.format("https://duguoxue.com/ershisishi/{0,number,#}.html",i))
            .forEach(url->download(url,e.str("name")));
        });
    }

}
